CreditCard Users Churn Prediction¶

df.shape

df['Income_Category'] = df['Income_Category'].astype('string')
df.Income_Category.str.strip()

pd.concat([df.head(), df.sample(5), df.tail()])

df.columns

df.info()

cat_cols = df.select_dtypes(include='object').columns
df[cat_cols] = df[cat_cols].astype('category')
df.dtypes

df.describe(include='category').T

df.describe(include='integer').T

df.describe(include='float').T

duplicated_df = df[df.duplicated()]
print(f"There are {duplicated_df.shape[0]} rows that are duplicated! We will drop these rows!")

missing_df = pd.DataFrame({
    "Missing": df.isnull().sum(),
    "Missing %": round((df.isnull().sum()/df.shape[0]*100), 2)
})
display(missing_df.sort_values(by='Missing', ascending=False))

for col in df.select_dtypes(include='category').columns:
    print(f"Feature: {col}")
    print("="*40)
    display(pd.DataFrame({"Counts": df[col].value_counts(dropna=False)}).sort_values(by='Counts', ascending=False))

df.select_dtypes('float').columns

histogram_boxplot(df.Credit_Limit)

histogram_boxplot(df.Avg_Open_To_Buy)

histogram_boxplot(df.Total_Amt_Chng_Q4_Q1)

histogram_boxplot(df.Total_Ct_Chng_Q4_Q1)

histogram_boxplot(df.Avg_Utilization_Ratio)

df.select_dtypes(include='integer').columns

histogram_boxplot(df.Customer_Age)

histogram_boxplot(df.Dependent_count)

histogram_boxplot(df.Months_on_book)

histogram_boxplot(df.Total_Relationship_Count)

histogram_boxplot(df.Months_Inactive_12_mon)

histogram_boxplot(df.Contacts_Count_12_mon)

histogram_boxplot(df.Total_Revolving_Bal)

histogram_boxplot(df.Total_Trans_Amt)

histogram_boxplot(df.Total_Trans_Ct)

df.select_dtypes(include='category').columns

perc_on_bar(df.Attrition_Flag)

perc_on_bar(df.Gender)

perc_on_bar(df.Education_Level)

perc_on_bar(df.Marital_Status)

perc_on_bar(df.Income_Category)

perc_on_bar(df.Card_Category)

plt.figure(figsize=(15,7))
sns.heatmap(df.corr(),annot=True,vmin=-1,vmax=1,fmt='.2f',cmap='coolwarm');

sns.pairplot(df, hue='Attrition_Flag');

cols = df.select_dtypes(include='float').columns.tolist()
show_boxplots(cols, 'Attrition_Flag')

cols = df.select_dtypes(include='integer').columns.tolist()
show_boxplots(cols, 'Attrition_Flag')

stacked_plot(df.Gender, df.Attrition_Flag)

stacked_plot(df.Education_Level, df.Attrition_Flag)

stacked_plot(df.Marital_Status, df.Attrition_Flag)

stacked_plot(df.Card_Category, df.Attrition_Flag)

df.nunique()

try:
    df.drop('CLIENTNUM', axis=1, inplace=True)
except:
    print('Already dropped')
df.head()

cols = ['Education_Level', 'Marital_Status', 'Card_Category', 'Income_Category', 'Attrition_Flag', 'Gender']
df[cols] = df[cols].astype('int')
df.info()

df.Income_Category.value_counts(dropna=False)

df.head()

# Separating target variable and other variables
X = df.drop(columns="Attrition_Flag")
X.head()

y = df["Attrition_Flag"]
y[:10]

# Splitting the data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=1, stratify=y)
print(X_train.shape, X_test.shape)

# Fit the model on original data i.e. before upsampling
log_reg = LogisticRegression()
log_reg.fit(X_train, y_train)
y_predict = log_reg.predict(X_test)
model_score = log_reg.score(X_test, y_test)
print(model_score)

get_metrics_score(log_reg)
make_confusion_matrix(log_reg, y_test)

array = df.values
X_up = array[:, 1:] # select all rows and first 8 columns which are the attributes
print(X_up)
y_up = array[:, 0]   # select all rows and the 8th column which is the classification "Yes", "No" for diabeties
y_up

test_size = 0.30 # taking 70:30 training and test set
seed = 7  # Random numbmer seeding for reapeatability of the code
X_train, X_test, y_train, y_test = train_test_split(X_up, y_up, test_size=test_size, random_state=seed)
type(X_train)

print(f"Before UpSampling, counts of label '1': {sum(y_train==1)}")
print(f"Before UpSampling, counts of label '0': {sum(y_train==0)} \n")

sm = SMOTE(sampling_strategy = 1 ,k_neighbors = 5, random_state=1)   #Synthetic Minority Over Sampling Technique
X_train_res, y_train_res = sm.fit_resample(X_train, y_train.ravel())

print(f"After UpSampling, counts of label '1': {sum(y_train_res==1)}")
print(f"After UpSampling, counts of label '0': {sum(y_train_res==0)} \n")

print(f'After UpSampling, the shape of train_X: {X_train_res.shape}')
print(f'After UpSampling, the shape of train_y: {y_train_res.shape} \n')

y_train_res

log_up = LogisticRegression()
log_up.fit(X_train_res, y_train_res)
y_predict = log_up.predict(X_test)
model_score = log_up.score(X_test, y_test)

#Calculating different metrics
get_metrics_score(log_up)

#Creating confusion matrix
make_confusion_matrix(log_up, y_test)

attrite_indices = df[df['Attrition_Flag'] == 0].index   
attrite = len(df[df['Attrition_Flag'] == 0])    
print(attrite)

existing_indices = df[df['Attrition_Flag'] == 1].index 
existing = len(df[df['Attrition_Flag'] == 1])  
print(existing)
print(existing - attrite)

df_down_sample = df.loc[down_sample_indices]  
df_down_sample.shape
df_down_sample.groupby(["Attrition_Flag"]).count()

array = df_down_sample.values
array

array = df_down_sample.values
X_down = array[:, 1:] # select all rows and first 8 columns which are the attributes
print(X_down)
y_down = array[:, 0]   # select all rows and the 8th column which is the classification "Yes", "No" for diabeties
y_down

test_size = 0.30 # taking 70:30 training and test set
seed = 7  # Random numbmer seeding for reapeatability of the code
X_train, X_test, y_train, y_test = train_test_split(X_down, y_down, test_size=test_size, random_state=seed)
type(X_train)

print(f'After DownSampling, the shape of X_train: {X_train.shape}')
print(f'After DownSampling, the shape of X_test: {X_test.shape}\n')

# Fit the model on 30%
log_down = LogisticRegression()
log_down.fit(X_train, y_train)
y_predict = log_down.predict(X_test)
model_score = log_down.score(X_test, y_test)
print(model_score)
print(metrics.confusion_matrix(y_test, y_predict))
print(metrics.classification_report(y_test, y_predict))

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1, stratify=y)
print(X_train.shape, X_test.shape)
print(y_train.shape, y_test.shape)

#Fitting the model
d_tree = DecisionTreeClassifier(random_state=1)
d_tree.fit(X_train,y_train)

#Calculating different metrics
get_metrics_score(d_tree)

#Creating confusion matrix
make_confusion_matrix(d_tree,y_test)

#Fitting the model
rf_estimator = RandomForestClassifier(random_state=1)
rf_estimator.fit(X_train,y_train)

#Calculating different metrics
get_metrics_score(rf_estimator)

#Creating confusion matrix
make_confusion_matrix(rf_estimator,y_test)

#Fitting the model
bagging_classifier = BaggingClassifier(random_state=1)
bagging_classifier.fit(X_train,y_train)

#Calculating different metrics
get_metrics_score(bagging_classifier)

#Creating confusion matrix
make_confusion_matrix(bagging_classifier,y_test)

#Fitting the model
gb_classifier = GradientBoostingClassifier(random_state=1)
gb_classifier.fit(X_train,y_train)

#Calculating different metrics
get_metrics_score(gb_classifier)

#Creating confusion matrix
make_confusion_matrix(gb_classifier,y_test)

abc = AdaBoostClassifier(random_state=1)
abc.fit(X_train, y_train)
get_metrics_score(abc)
make_confusion_matrix(abc, y_test)

#Fitting the model
xgb_classifier = XGBClassifier(random_state=1, eval_metric='logloss')
xgb_classifier.fit(X_train, y_train)

#Calculating different metrics
get_metrics_score(xgb_classifier)

#Creating confusion matrix
make_confusion_matrix(xgb_classifier, y_test)

# defining list of models
models = [log_reg, log_up, log_down, d_tree, rf_estimator,
          bagging_classifier, abc, gb_classifier, xgb_classifier]

model_names = [
    'Logistic Regression', 'Logistic Regression Upsample', 'Logistic Regression Downsample',                              
     'Decision Tree', 'Random Forest', 'Bagging Classifier', 'AdaBoost Classifier', 
    'Gradient Boosting Classifier','XGBoost Classifier']

show_model_performance(models, model_names)

%%time 

#Creating pipeline
pipe=make_pipeline(StandardScaler(), XGBClassifier(random_state=1,eval_metric='logloss'))

#Parameter grid to pass in GridSearchCV
param_grid={'xgbclassifier__n_estimators':np.arange(50, 200, 50),'xgbclassifier__scale_pos_weight':[0,1,2],
            'xgbclassifier__learning_rate':[0.05, 0.1, 0.2], 'xgbclassifier__gamma':[0,1,3],
            'xgbclassifier__subsample':[0.7, 0.9, 1]}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.f1_score)

# Run the grid search
grid_obj = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring=scorer, cv=5, n_jobs=-1)
grid_obj.fit(X_train, y_train)

print(f"Best Parameters:{grid_obj.best_params_} \nScore: {grid_obj.best_score_}")

# Creating new pipeline with best parameters
xgb_tuned1 = make_pipeline(
    StandardScaler(),
    XGBClassifier(
        random_state=1,
        n_estimators=200,
        scale_pos_weight=1,
        subsample=0.7,
        learning_rate=0.2,
        gamma=0,
        eval_metric='logloss',
        n_jobs=-1
    ),
)

# Fit the model on training data
xgb_tuned1.fit(X_train, y_train)
#Calculating different metrics
get_metrics_score(xgb_tuned1)

#Creating confusion matrix
make_confusion_matrix(xgb_tuned1, y_test)

%%time 

#Creating pipeline
pipe=make_pipeline(StandardScaler(),XGBClassifier(random_state=1,eval_metric='logloss', n_estimators=50))

#Parameter grid to pass in RandomizedSearchCV
param_grid={'xgbclassifier__n_estimators':np.arange(50,300,50),
            'xgbclassifier__scale_pos_weight':[0,1,2,5,10],
            'xgbclassifier__learning_rate':[0.01,0.1,0.2,0.05],
            'xgbclassifier__gamma':[0,1,3,5],
            'xgbclassifier__subsample':[0.7,0.8,0.9,1],
            'xgbclassifier__max_depth':np.arange(1,10,1),
            'xgbclassifier__reg_lambda':[0,1,2,5,10]}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.recall_score)

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(estimator=pipe, param_distributions=param_grid, n_iter=50, 
                                   scoring=scorer, cv=5, random_state=1, n_jobs=-1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(X_train,y_train)

print(f"Best Parameters:{randomized_cv.best_params_} \nScore: {randomized_cv.best_score_}")

# Creating new pipeline with best parameters
xgb_tuned2 = make_pipeline(
    StandardScaler(),
    XGBClassifier(
        random_state=1,
        n_estimators=50,
        scale_pos_weight=10,
        reg_lambda=2,
        max_depth=2,
        subsample=0.8,
        learning_rate=0.05,
        gamma=3,
        eval_metric='logloss',
        n_jobs=-1
    ),
)

# Fit the model on training data
xgb_tuned2.fit(X_train, y_train)
#Calculating different metrics
get_metrics_score(xgb_tuned2)

#Creating confusion matrix
make_confusion_matrix(xgb_tuned2, y_test)

%%time 

#Creating pipeline
pipe=make_pipeline(StandardScaler(), GradientBoostingClassifier(init=AdaBoostClassifier(random_state=1), random_state=1))

#Parameter grid to pass in GridSearchCV
param_grid={
    'gradientboostingclassifier__n_estimators':np.arange(100, 300, 50),
    'gradientboostingclassifier__subsample':[0.5, 0.6, 0.7, 0.8, 0.9, 1],
    'gradientboostingclassifier__max_features':[0.7, 0.8, 0.9, 1]
}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.f1_score)

# Run the grid search
grid_obj = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring=scorer, cv=5, n_jobs=-1)
grid_obj.fit(X_train, y_train)

print("Best parameters are {} with CV score={}:" .format(grid_obj.best_params_, grid_obj.best_score_))

# Creating new pipeline with best parameters
gbc_tuned1 = make_pipeline(
    StandardScaler(),
    GradientBoostingClassifier(
        init=AdaBoostClassifier(random_state=1),
        random_state=1,
        n_estimators=250,
        subsample=1,
        max_features=0.8,
    ),
)

# Fit the model on training data
gbc_tuned1.fit(X_train, y_train)
# Calculating different metrics
get_metrics_score(gbc_tuned1)

# Creating confusion matrix
make_confusion_matrix(gbc_tuned1, y_test)

gbc_tuned1.get_params()

%%time 

#Creating pipeline
pipe=make_pipeline(StandardScaler(), GradientBoostingClassifier(init=AdaBoostClassifier(random_state=1), random_state=1))

#Parameter grid to pass in GridSearchCV
param_grid={
    'gradientboostingclassifier__learning_rate': np.arange(0.1, 1, 0.2),
    'gradientboostingclassifier__max_depth': np.arange(1, 10, 2),
    'gradientboostingclassifier__max_features': [0.5, 0.7, 0.8, 1],
    'gradientboostingclassifier__n_estimators': np.arange(50, 300, 50),
    'gradientboostingclassifier__subsample':[0.5, 0.6, 0.7, 0.8, 0.9, 1],
    'gradientboostingclassifier__max_features':[0.7, 0.8, 0.9, 1]
}    
    
# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.recall_score)

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(estimator=pipe, param_distributions=param_grid, n_iter=50, 
                                   scoring=scorer, cv=5, random_state=1, n_jobs=-1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(X_train,y_train)

print(f"Best Parameters:{randomized_cv.best_params_} \nScore: {randomized_cv.best_score_}")

# Creating new pipeline with best parameters
gbc_tuned2 = make_pipeline(
    StandardScaler(),
    GradientBoostingClassifier(
        init=AdaBoostClassifier(random_state=1),
        subsample=0.8,
        n_estimators=100,
        max_features=1,
        max_depth=1,
        learning_rate=0.1
    ),
)

# Fit the model on training data
gbc_tuned2.fit(X_train, y_train)
# Calculating different metrics
get_metrics_score(gbc_tuned2)

# Creating confusion matrix
make_confusion_matrix(gbc_tuned2, y_test)

%%time 

#Creating pipeline
pipe=make_pipeline(StandardScaler(), 
                   RandomForestClassifier(
                       class_weight={0: 0.80, 1: 0.20},
                       random_state=1,
                       oob_score=True,
                       bootstrap=True
                   )
                  )

#Parameter grid to pass in GridSearchCV
param_grid={
    'randomforestclassifier__max_depth': np.arange(5, 30, 5),
    'randomforestclassifier__max_features': ['sqrt','log2', None],
    'randomforestclassifier__min_samples_leaf': np.arange(1, 15, 5),
    'randomforestclassifier__min_samples_split': np.arange(2, 20, 5),
    'randomforestclassifier__n_estimators': np.arange(10, 110, 10)}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.f1_score)

# Run the grid search
grid_obj = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring=scorer, cv=5, n_jobs=-1)
grid_obj.fit(X_train, y_train)

print(f"Best Parameters:{grid_obj.best_params_} \nScore: {grid_obj.best_score_}")